{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:40.829714Z",
     "start_time": "2021-10-12T08:06:39.671676Z"
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import os\n",
    "import re\n",
    "import gc\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('max_columns', None)\n",
    "pd.set_option('max_rows', 200)\n",
    "pd.set_option('float_format', lambda x: '%.3f' % x)\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:40.914118Z",
     "start_time": "2021-10-12T08:06:40.831783Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15016, 19)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>op_date</th>\n",
       "      <th>user_name</th>\n",
       "      <th>action</th>\n",
       "      <th>auth_type</th>\n",
       "      <th>ip</th>\n",
       "      <th>ip_location_type_keyword</th>\n",
       "      <th>ip_risk_level</th>\n",
       "      <th>location</th>\n",
       "      <th>client_type</th>\n",
       "      <th>browser_source</th>\n",
       "      <th>device_model</th>\n",
       "      <th>os_type</th>\n",
       "      <th>os_version</th>\n",
       "      <th>browser_type</th>\n",
       "      <th>browser_version</th>\n",
       "      <th>bus_system_code</th>\n",
       "      <th>op_target</th>\n",
       "      <th>risk_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>access:test_d:20180101111639:bBp1</td>\n",
       "      <td>2018/1/1 11:16</td>\n",
       "      <td>test_d</td>\n",
       "      <td>login</td>\n",
       "      <td>otp</td>\n",
       "      <td>192.168.100.101</td>\n",
       "      <td>内网</td>\n",
       "      <td>1级</td>\n",
       "      <td>{\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>think_pad_e460</td>\n",
       "      <td>windows</td>\n",
       "      <td>windows 10</td>\n",
       "      <td>chrome</td>\n",
       "      <td>chrome 90</td>\n",
       "      <td>coremail</td>\n",
       "      <td>management</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>access:test_d:20180101121524:OBSg</td>\n",
       "      <td>2018/1/1 12:15</td>\n",
       "      <td>test_d</td>\n",
       "      <td>login</td>\n",
       "      <td>qr</td>\n",
       "      <td>192.168.100.101</td>\n",
       "      <td>内网</td>\n",
       "      <td>1级</td>\n",
       "      <td>{\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>think_pad_e460</td>\n",
       "      <td>windows</td>\n",
       "      <td>windows 10</td>\n",
       "      <td>edge</td>\n",
       "      <td>edge 93</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>access:test_d:20180101151333:BpQN</td>\n",
       "      <td>2018/1/1 15:13</td>\n",
       "      <td>test_d</td>\n",
       "      <td>login</td>\n",
       "      <td>qr</td>\n",
       "      <td>192.168.100.101</td>\n",
       "      <td>内网</td>\n",
       "      <td>1级</td>\n",
       "      <td>{\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>think_pad_e460</td>\n",
       "      <td>windows</td>\n",
       "      <td>windows 10</td>\n",
       "      <td>chrome</td>\n",
       "      <td>chrome 90</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>access:test_d:20180101124502:hYQm</td>\n",
       "      <td>2018/1/1 12:45</td>\n",
       "      <td>test_d</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>192.168.100.101</td>\n",
       "      <td>内网</td>\n",
       "      <td>1级</td>\n",
       "      <td>{\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>think_pad_e460</td>\n",
       "      <td>windows</td>\n",
       "      <td>windows 10</td>\n",
       "      <td>edge</td>\n",
       "      <td>edge 93</td>\n",
       "      <td>oa</td>\n",
       "      <td>management</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>access:test_d:20180101202749:FkDK</td>\n",
       "      <td>2018/1/1 20:27</td>\n",
       "      <td>test_d</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>192.168.100.101</td>\n",
       "      <td>内网</td>\n",
       "      <td>1级</td>\n",
       "      <td>{\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>think_pad_e460</td>\n",
       "      <td>windows</td>\n",
       "      <td>windows 10</td>\n",
       "      <td>edge</td>\n",
       "      <td>edge 93</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          session_id         op_date user_name action  \\\n",
       "0  access:test_d:20180101111639:bBp1  2018/1/1 11:16    test_d  login   \n",
       "1  access:test_d:20180101121524:OBSg  2018/1/1 12:15    test_d  login   \n",
       "2  access:test_d:20180101151333:BpQN  2018/1/1 15:13    test_d  login   \n",
       "3  access:test_d:20180101124502:hYQm  2018/1/1 12:45    test_d    sso   \n",
       "4  access:test_d:20180101202749:FkDK  2018/1/1 20:27    test_d    sso   \n",
       "\n",
       "  auth_type               ip ip_location_type_keyword ip_risk_level  \\\n",
       "0       otp  192.168.100.101                       内网            1级   \n",
       "1        qr  192.168.100.101                       内网            1级   \n",
       "2        qr  192.168.100.101                       内网            1级   \n",
       "3       NaN  192.168.100.101                       内网            1级   \n",
       "4       NaN  192.168.100.101                       内网            1级   \n",
       "\n",
       "                                            location client_type  \\\n",
       "0  {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...         web   \n",
       "1  {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...         web   \n",
       "2  {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...         web   \n",
       "3  {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...         web   \n",
       "4  {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl...         web   \n",
       "\n",
       "  browser_source    device_model  os_type  os_version browser_type  \\\n",
       "0        desktop  think_pad_e460  windows  windows 10       chrome   \n",
       "1        desktop  think_pad_e460  windows  windows 10         edge   \n",
       "2        desktop  think_pad_e460  windows  windows 10       chrome   \n",
       "3        desktop  think_pad_e460  windows  windows 10         edge   \n",
       "4        desktop  think_pad_e460  windows  windows 10         edge   \n",
       "\n",
       "  browser_version bus_system_code   op_target  risk_label  \n",
       "0       chrome 90        coremail  management           0  \n",
       "1         edge 93      order-mgnt       sales           0  \n",
       "2       chrome 90      order-mgnt       sales           0  \n",
       "3         edge 93              oa  management           0  \n",
       "4         edge 93      order-mgnt       sales           0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('raw_data/train_dataset.csv', sep='\\t')\n",
    "print(train.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.027901Z",
     "start_time": "2021-10-12T08:06:40.916126Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 18)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>op_date</th>\n",
       "      <th>user_name</th>\n",
       "      <th>action</th>\n",
       "      <th>auth_type</th>\n",
       "      <th>ip</th>\n",
       "      <th>ip_location_type_keyword</th>\n",
       "      <th>ip_risk_level</th>\n",
       "      <th>location</th>\n",
       "      <th>client_type</th>\n",
       "      <th>browser_source</th>\n",
       "      <th>device_model</th>\n",
       "      <th>os_type</th>\n",
       "      <th>os_version</th>\n",
       "      <th>browser_type</th>\n",
       "      <th>browser_version</th>\n",
       "      <th>bus_system_code</th>\n",
       "      <th>op_target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>access:test_c:20191023212545:H2in</td>\n",
       "      <td>2019/10/23 21:25</td>\n",
       "      <td>test_c</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.10.135.254</td>\n",
       "      <td>代理IP</td>\n",
       "      <td>3级</td>\n",
       "      <td>{\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>macbook</td>\n",
       "      <td>macOS</td>\n",
       "      <td>macOS Big Sur 11</td>\n",
       "      <td>safari</td>\n",
       "      <td>safari 13</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>access:test_c:20191023095634:ylxO</td>\n",
       "      <td>2019/10/23 9:56</td>\n",
       "      <td>test_c</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.10.135.254</td>\n",
       "      <td>代理IP</td>\n",
       "      <td>3级</td>\n",
       "      <td>{\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>macbook</td>\n",
       "      <td>macOS</td>\n",
       "      <td>macOS Big Sur 11</td>\n",
       "      <td>safari</td>\n",
       "      <td>safari 13</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>access:test_c:20191023104233:tc9Y</td>\n",
       "      <td>2019/10/23 10:42</td>\n",
       "      <td>test_c</td>\n",
       "      <td>login</td>\n",
       "      <td>sms</td>\n",
       "      <td>27.10.135.254</td>\n",
       "      <td>代理IP</td>\n",
       "      <td>3级</td>\n",
       "      <td>{\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>macbook</td>\n",
       "      <td>macOS</td>\n",
       "      <td>macOS Big Sur 11</td>\n",
       "      <td>safari</td>\n",
       "      <td>safari 13</td>\n",
       "      <td>order-mgnt</td>\n",
       "      <td>sales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>access:test_c:20191023142416:8rjC</td>\n",
       "      <td>2019/10/23 14:24</td>\n",
       "      <td>test_c</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.10.135.254</td>\n",
       "      <td>代理IP</td>\n",
       "      <td>3级</td>\n",
       "      <td>{\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>macbook</td>\n",
       "      <td>macOS</td>\n",
       "      <td>macOS Big Sur 11</td>\n",
       "      <td>safari</td>\n",
       "      <td>safari 13</td>\n",
       "      <td>coremail</td>\n",
       "      <td>management</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>access:test_c:20191023210513:cOCi</td>\n",
       "      <td>2019/10/23 21:05</td>\n",
       "      <td>test_c</td>\n",
       "      <td>sso</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.10.135.254</td>\n",
       "      <td>代理IP</td>\n",
       "      <td>3级</td>\n",
       "      <td>{\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...</td>\n",
       "      <td>web</td>\n",
       "      <td>desktop</td>\n",
       "      <td>macbook</td>\n",
       "      <td>macOS</td>\n",
       "      <td>macOS Big Sur 11</td>\n",
       "      <td>safari</td>\n",
       "      <td>safari 13</td>\n",
       "      <td>reimbursement</td>\n",
       "      <td>finance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          session_id           op_date user_name action  \\\n",
       "0  access:test_c:20191023212545:H2in  2019/10/23 21:25    test_c    sso   \n",
       "1  access:test_c:20191023095634:ylxO   2019/10/23 9:56    test_c    sso   \n",
       "2  access:test_c:20191023104233:tc9Y  2019/10/23 10:42    test_c  login   \n",
       "3  access:test_c:20191023142416:8rjC  2019/10/23 14:24    test_c    sso   \n",
       "4  access:test_c:20191023210513:cOCi  2019/10/23 21:05    test_c    sso   \n",
       "\n",
       "  auth_type             ip ip_location_type_keyword ip_risk_level  \\\n",
       "0       NaN  27.10.135.254                     代理IP            3级   \n",
       "1       NaN  27.10.135.254                     代理IP            3级   \n",
       "2       sms  27.10.135.254                     代理IP            3级   \n",
       "3       NaN  27.10.135.254                     代理IP            3级   \n",
       "4       NaN  27.10.135.254                     代理IP            3级   \n",
       "\n",
       "                                            location client_type  \\\n",
       "0  {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...         web   \n",
       "1  {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...         web   \n",
       "2  {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...         web   \n",
       "3  {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...         web   \n",
       "4  {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":...         web   \n",
       "\n",
       "  browser_source device_model os_type        os_version browser_type  \\\n",
       "0        desktop      macbook   macOS  macOS Big Sur 11       safari   \n",
       "1        desktop      macbook   macOS  macOS Big Sur 11       safari   \n",
       "2        desktop      macbook   macOS  macOS Big Sur 11       safari   \n",
       "3        desktop      macbook   macOS  macOS Big Sur 11       safari   \n",
       "4        desktop      macbook   macOS  macOS Big Sur 11       safari   \n",
       "\n",
       "  browser_version bus_system_code   op_target  \n",
       "0       safari 13      order-mgnt       sales  \n",
       "1       safari 13      order-mgnt       sales  \n",
       "2       safari 13      order-mgnt       sales  \n",
       "3       safari 13        coremail  management  \n",
       "4       safari 13   reimbursement     finance  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('raw_data/test_dataset.csv', sep='\\t')\n",
    "print(test.shape)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.034735Z",
     "start_time": "2021-10-12T08:06:41.029663Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    12076\n",
       "1     2940\n",
       "Name: risk_label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['risk_label'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.223308Z",
     "start_time": "2021-10-12T08:06:41.036209Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_name test_d 0.190810465858328\n",
      "user_name test_c 0.2004201680672269\n",
      "user_name test_a 0.19375305026842363\n",
      "user_name test_b 0.20043763676148796\n",
      "user_name test_g 0.195578231292517\n",
      "user_name test_e 0.1988888888888889\n",
      "user_name test_f 0.19234116623150566\n",
      "==================================================\n",
      "action login 0.1932896671567972\n",
      "action sso 0.19827471798274718\n",
      "==================================================\n",
      "auth_type otp 0.19203491543917076\n",
      "auth_type qr 0.1888772298006296\n",
      "auth_type nan nan\n",
      "auth_type sms 0.19239013933547697\n",
      "auth_type pwd 0.19989339019189764\n",
      "==================================================\n",
      "ip 192.168.100.101 0.19682539682539682\n",
      "ip 14.196.145.66 0.18600867678958785\n",
      "ip 27.10.135.254 0.1939799331103679\n",
      "ip 192.168.100.103 0.20709105560032232\n",
      "ip 192.168.0.100 0.18235294117647058\n",
      "==================================================\n",
      "ip_location_type_keyword 内网 0.19747828991315966\n",
      "ip_location_type_keyword 家庭宽带 0.18600867678958785\n",
      "ip_location_type_keyword 代理IP 0.1939799331103679\n",
      "==================================================\n",
      "ip_risk_level 1级 0.19792024750773463\n",
      "ip_risk_level 2级 0.18543956043956045\n",
      "ip_risk_level 3级 0.1939799331103679\n",
      "==================================================\n",
      "location {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl\":\"销售部\"} 0.19682539682539682\n",
      "location {\"first_lvl\":\"四川省\",\"sec_lvl\":\"成都市\",\"third_lvl\":\"武侯区\"} 0.18600867678958785\n",
      "location {\"first_lvl\":\"重庆\",\"sec_lvl\":\"重庆市\",\"third_lvl\":\"江北区\"} 0.1939799331103679\n",
      "location {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"9楼\",\"third_lvl\":\"会议室\"} 0.20709105560032232\n",
      "location {\"first_lvl\":\"成都分公司\",\"sec_lvl\":\"10楼\",\"third_lvl\":\"会议室\"} 0.18235294117647058\n",
      "==================================================\n",
      "client_type web 0.19579115610015982\n",
      "==================================================\n",
      "browser_source desktop 0.19579115610015982\n",
      "==================================================\n",
      "device_model think_pad_e460 0.19792024750773463\n",
      "device_model macbook 0.19131679389312978\n",
      "device_model think_pad_t480 0.1843220338983051\n",
      "device_model think_pad_l470 0.18235294117647058\n",
      "==================================================\n",
      "os_type windows 0.1965170278637771\n",
      "os_type macOS 0.19131679389312978\n",
      "==================================================\n",
      "os_version windows 10 0.19792024750773463\n",
      "os_version macOS Big Sur 11 0.19131679389312978\n",
      "os_version windows 11 0.1843220338983051\n",
      "os_version windows 7 0.18235294117647058\n",
      "==================================================\n",
      "browser_type chrome 0.2008506616257089\n",
      "browser_type edge 0.1934199134199134\n",
      "browser_type safari 0.19131679389312978\n",
      "browser_type firefox 0.1834862385321101\n",
      "browser_type ie 0.1851063829787234\n",
      "==================================================\n",
      "browser_version chrome 90 0.20235454700563044\n",
      "browser_version edge 93 0.1934199134199134\n",
      "browser_version safari 13 0.19131679389312978\n",
      "browser_version firefox 78 0.1834862385321101\n",
      "browser_version ie 11 0.17785234899328858\n",
      "browser_version chrome 93 0.19122257053291536\n",
      "browser_version chrome 77 0.16666666666666666\n",
      "browser_version ie 9 0.19767441860465115\n",
      "==================================================\n",
      "bus_system_code coremail 0.18959668300037694\n",
      "bus_system_code order-mgnt 0.19411146161934806\n",
      "bus_system_code oa 0.1900328587075575\n",
      "bus_system_code crm 0.19945541184479237\n",
      "bus_system_code reimbursement 0.20221606648199447\n",
      "bus_system_code salary 0.21132075471698114\n",
      "bus_system_code attendance 0.2052863436123348\n",
      "==================================================\n",
      "op_target management 0.18977450323732975\n",
      "op_target sales 0.19615234628883402\n",
      "op_target finance 0.20221606648199447\n",
      "op_target hr 0.20642857142857143\n",
      "==================================================\n"
     ]
    }
   ],
   "source": [
    "for f in ['user_name', 'action', 'auth_type', 'ip',\n",
    "          'ip_location_type_keyword', 'ip_risk_level', 'location', 'client_type',\n",
    "          'browser_source', 'device_model', 'os_type', 'os_version',\n",
    "          'browser_type', 'browser_version', 'bus_system_code', 'op_target']:\n",
    "    for v in train[f].unique():\n",
    "        print(f, v, train[train[f] == v]['risk_label'].mean())\n",
    "    print('='*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.234311Z",
     "start_time": "2021-10-12T08:06:41.224690Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25016, 19)\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([train, test])\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.729739Z",
     "start_time": "2021-10-12T08:06:41.235882Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d472bc4bf7e543fa91d1137f8385d8b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data['location_first_lvl'] = data['location'].astype(str).apply(lambda x: json.loads(x)['first_lvl'])\n",
    "data['location_sec_lvl'] = data['location'].astype(str).apply(lambda x: json.loads(x)['sec_lvl'])\n",
    "data['location_third_lvl'] = data['location'].astype(str).apply(lambda x: json.loads(x)['third_lvl'])\n",
    "\n",
    "data.drop(['client_type', 'browser_source'], axis=1, inplace=True)\n",
    "data['auth_type'].fillna('__NaN__', inplace=True)\n",
    "\n",
    "for col in tqdm(['user_name', 'action', 'auth_type', 'ip', \n",
    "                 'ip_location_type_keyword', 'ip_risk_level', 'location', 'device_model',\n",
    "                 'os_type', 'os_version', 'browser_type', 'browser_version',\n",
    "                 'bus_system_code', 'op_target', 'location_first_lvl', 'location_sec_lvl', \n",
    "                 'location_third_lvl']):\n",
    "    lbl = LabelEncoder()\n",
    "    data[col] = lbl.fit_transform(data[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.763052Z",
     "start_time": "2021-10-12T08:06:41.732117Z"
    }
   },
   "outputs": [],
   "source": [
    "data['op_date'] = pd.to_datetime(data['op_date'])\n",
    "data['op_ts'] = data[\"op_date\"].values.astype(np.int64) // 10 ** 9\n",
    "data = data.sort_values(by=['user_name', 'op_ts']).reset_index(drop=True)\n",
    "data['last_ts'] = data.groupby(['user_name'])['op_ts'].shift(1)\n",
    "data['ts_diff1'] = data['op_ts'] - data['last_ts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.779435Z",
     "start_time": "2021-10-12T08:06:41.764845Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for f in ['ip', 'location', 'device_model', 'os_version', 'browser_version']:\n",
    "    data[f'user_{f}_nunique'] = data.groupby(['user_name'])[f].transform('nunique')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.790331Z",
     "start_time": "2021-10-12T08:06:41.780710Z"
    }
   },
   "outputs": [],
   "source": [
    "for method in ['mean', 'max', 'min', 'std']:\n",
    "    data[f'ts_diff1_{method}'] = data.groupby('user_name')['ts_diff1'].transform(method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:41.806046Z",
     "start_time": "2021-10-12T08:06:41.791599Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15016, 32) (10000, 32)\n"
     ]
    }
   ],
   "source": [
    "train = data[data['risk_label'].notna()]\n",
    "test = data[data['risk_label'].isna()]\n",
    "\n",
    "print(train.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:43.438497Z",
     "start_time": "2021-10-12T08:06:41.807507Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fold_1 Training ================================\n",
      "\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[12]\ttrain's auc: 0.648341\tvalid's auc: 0.51527\n",
      "\n",
      "Fold_2 Training ================================\n",
      "\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[1]\ttrain's auc: 0.563305\tvalid's auc: 0.524088\n",
      "\n",
      "Fold_3 Training ================================\n",
      "\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[3]\ttrain's auc: 0.602498\tvalid's auc: 0.517259\n",
      "\n",
      "Fold_4 Training ================================\n",
      "\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[2]\ttrain's auc: 0.576999\tvalid's auc: 0.512423\n",
      "\n",
      "Fold_5 Training ================================\n",
      "\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "Early stopping, best iteration is:\n",
      "[15]\ttrain's auc: 0.656338\tvalid's auc: 0.522001\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ts_diff1</td>\n",
       "      <td>50.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>op_ts</td>\n",
       "      <td>41.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bus_system_code</td>\n",
       "      <td>20.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>auth_type</td>\n",
       "      <td>16.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>browser_version</td>\n",
       "      <td>9.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ts_diff1_mean</td>\n",
       "      <td>6.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>user_name</td>\n",
       "      <td>6.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>op_target</td>\n",
       "      <td>6.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>location_third_lvl</td>\n",
       "      <td>5.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ip</td>\n",
       "      <td>5.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>action</td>\n",
       "      <td>4.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ts_diff1_std</td>\n",
       "      <td>4.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>browser_type</td>\n",
       "      <td>4.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>device_model</td>\n",
       "      <td>4.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ts_diff1_max</td>\n",
       "      <td>3.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>user_ip_nunique</td>\n",
       "      <td>3.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ip_risk_level</td>\n",
       "      <td>2.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ip_location_type_keyword</td>\n",
       "      <td>2.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>os_version</td>\n",
       "      <td>1.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>location</td>\n",
       "      <td>1.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>location_sec_lvl</td>\n",
       "      <td>1.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>user_location_nunique</td>\n",
       "      <td>1.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>os_type</td>\n",
       "      <td>0.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>user_os_version_nunique</td>\n",
       "      <td>0.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>user_browser_version_nunique</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>location_first_lvl</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>user_device_model_nunique</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ts_diff1_min</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          column  importance\n",
       "0                       ts_diff1      50.400\n",
       "1                          op_ts      41.200\n",
       "2                bus_system_code      20.000\n",
       "3                      auth_type      16.400\n",
       "4                browser_version       9.000\n",
       "5                  ts_diff1_mean       6.800\n",
       "6                      user_name       6.600\n",
       "7                      op_target       6.400\n",
       "8             location_third_lvl       5.600\n",
       "9                             ip       5.400\n",
       "10                        action       4.800\n",
       "11                  ts_diff1_std       4.800\n",
       "12                  browser_type       4.200\n",
       "13                  device_model       4.200\n",
       "14                  ts_diff1_max       3.200\n",
       "15               user_ip_nunique       3.000\n",
       "16                 ip_risk_level       2.200\n",
       "17      ip_location_type_keyword       2.000\n",
       "18                    os_version       1.400\n",
       "19                      location       1.400\n",
       "20              location_sec_lvl       1.200\n",
       "21         user_location_nunique       1.200\n",
       "22                       os_type       0.600\n",
       "23       user_os_version_nunique       0.400\n",
       "24  user_browser_version_nunique       0.000\n",
       "25            location_first_lvl       0.000\n",
       "26     user_device_model_nunique       0.000\n",
       "27                  ts_diff1_min       0.000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ycol = 'risk_label'\n",
    "feature_names = list(\n",
    "    filter(lambda x: x not in [ycol, 'session_id', 'op_date', 'last_ts'], train.columns))\n",
    "\n",
    "model = lgb.LGBMClassifier(objective='binary',\n",
    "                           boosting_type='gbdt',\n",
    "                           tree_learner='serial',\n",
    "                           num_leaves=32,\n",
    "                           max_depth=6,\n",
    "                           learning_rate=0.1,\n",
    "                           n_estimators=10000,\n",
    "                           subsample=0.8,\n",
    "                           feature_fraction=0.6,\n",
    "                           reg_alpha=0.,\n",
    "                           reg_lambda=0.,\n",
    "                           random_state=1983,\n",
    "                           is_unbalance=True,\n",
    "                           metric='auc')\n",
    "\n",
    "\n",
    "oof = []\n",
    "prediction = test[['session_id']]\n",
    "prediction[ycol] = 0\n",
    "df_importance_list = []\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=1983)\n",
    "for fold_id, (trn_idx, val_idx) in enumerate(kfold.split(train[feature_names], train[ycol])):\n",
    "    X_train = train.iloc[trn_idx][feature_names]\n",
    "    Y_train = train.iloc[trn_idx][ycol]\n",
    "\n",
    "    X_val = train.iloc[val_idx][feature_names]\n",
    "    Y_val = train.iloc[val_idx][ycol]\n",
    "\n",
    "    print('\\nFold_{} Training ================================\\n'.format(fold_id+1))\n",
    "\n",
    "    lgb_model = model.fit(X_train,\n",
    "                          Y_train,\n",
    "                          eval_names=['train', 'valid'],\n",
    "                          eval_set=[(X_train, Y_train), (X_val, Y_val)],\n",
    "                          verbose=500,\n",
    "                          eval_metric='auc',\n",
    "                          early_stopping_rounds=50)\n",
    "\n",
    "    pred_val = lgb_model.predict_proba(\n",
    "        X_val, num_iteration=lgb_model.best_iteration_)\n",
    "    df_oof = train.iloc[val_idx][['session_id', ycol]].copy()\n",
    "    df_oof['pred'] = pred_val[:, 1]\n",
    "    oof.append(df_oof)\n",
    "\n",
    "    pred_test = lgb_model.predict_proba(\n",
    "        test[feature_names], num_iteration=lgb_model.best_iteration_)\n",
    "    prediction[ycol] += pred_test[:, 1] / kfold.n_splits\n",
    "\n",
    "    df_importance = pd.DataFrame({\n",
    "        'column': feature_names,\n",
    "        'importance': lgb_model.feature_importances_,\n",
    "    })\n",
    "    df_importance_list.append(df_importance)\n",
    "\n",
    "    del lgb_model, pred_val, pred_test, X_train, Y_train, X_val, Y_val\n",
    "    gc.collect()\n",
    "    \n",
    "    \n",
    "df_importance = pd.concat(df_importance_list)\n",
    "df_importance = df_importance.groupby(['column'])['importance'].agg(\n",
    "    'mean').sort_values(ascending=False).reset_index()\n",
    "df_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:43.457038Z",
     "start_time": "2021-10-12T08:06:43.440662Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roc_auc_score 0.5070717091076244\n"
     ]
    }
   ],
   "source": [
    "df_oof = pd.concat(oof)\n",
    "print('roc_auc_score', roc_auc_score(df_oof[ycol], df_oof['pred']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:43.473879Z",
     "start_time": "2021-10-12T08:06:43.459072Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>ret</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6147</th>\n",
       "      <td>1</td>\n",
       "      <td>0.324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6148</th>\n",
       "      <td>2</td>\n",
       "      <td>0.320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6149</th>\n",
       "      <td>3</td>\n",
       "      <td>0.324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6150</th>\n",
       "      <td>4</td>\n",
       "      <td>0.335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6151</th>\n",
       "      <td>5</td>\n",
       "      <td>0.320</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id   ret\n",
       "6147   1 0.324\n",
       "6148   2 0.320\n",
       "6149   3 0.324\n",
       "6150   4 0.335\n",
       "6151   5 0.320"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction['id'] = range(len(prediction))\n",
    "prediction['id'] = prediction['id'] + 1\n",
    "prediction = prediction[['id', 'risk_label']].copy()\n",
    "prediction.columns = ['id', 'ret']\n",
    "prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:43.478384Z",
     "start_time": "2021-10-12T08:06:43.475930Z"
    }
   },
   "outputs": [],
   "source": [
    "# prediction['rank'] = prediction['risk_label'].rank()\n",
    "# prediction['ret'] = 0\n",
    "# prediction.loc[prediction['rank'] <= int(prediction.shape[0] * train['risk_label'].mean()), 'ret'] = 1\n",
    "\n",
    "# prediction = prediction[['session_id', 'ret']].copy()\n",
    "# prediction.columns = ['id', 'ret']\n",
    "# prediction['id'] = range(len(prediction))\n",
    "# prediction['id'] = prediction['id'] + 1\n",
    "# prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T08:06:43.538349Z",
     "start_time": "2021-10-12T08:06:43.480511Z"
    }
   },
   "outputs": [],
   "source": [
    "# print(prediction['ret'].value_counts())\n",
    "prediction.to_csv('bottomline.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
